source('../settings/settings.R')
source('commonFunctions.R')
drive1 <- read.csv('../data/processed/analysis/TT1_Drive_1_PP.csv')
drive2 <- read.csv('../data/processed/Analysis/TT1_Drive_2_PP.csv')
drive3 <- read.csv('../data/processed/Analysis/TT1_Drive_3_PP.csv')
drive4 <- read.csv('../data/processed/Analysis/TT1_Drive_4_PP.csv', stringsAsFactors = T)
set.seed(43)
combinedDf <- cbind(drive4,
drive1$MeanPP_Seg0,
drive2$MeanPP, drive3$MeanPP,
drive2$StdPP, drive3$StdPP,
drive2$MeanPP_SegMax, drive3$MeanPP_SegMax,
drive2$MeanPP_Seg0, drive3$MeanPP_Seg0,
drive2$StdPP_SegMax, drive3$StdPP_SegMax,
drive2$StdPP_Seg0, drive3$StdPP_Seg0,
drive2$MeanPP_AccHigh, drive3$MeanPP_AccHigh,
drive2$X.MeanPP_AccLow, drive3$X.MeanPP_AccLow,
drive2$StdPP_AccHigh, drive3$StdPP_AccHigh,
drive2$StdPP_AccLow, drive3$StdPP_AccLow
)
names(combinedDf) <- c(names(drive4),
"PP_Dev_1_Turning",
"PP_Dev_2", "PP_Dev_3",
"Std_PP_2", "Std_PP_3",
"PP_Dev_2_Straight", "PP_Dev_3_Straight",
"PP_Dev_2_Turning", "PP_Dev_3_Turning",
"Std_PP_2_Straight", "Std_PP_3_Straight",
"Std_PP_2_Turning", "Std_PP_3_Turning",
"Mean_PP_2_AccHigh", "Mean_PP_3_AccHigh",
"Mean_PP_2_AccLow", "Mean_PP_3_AccLow",
"Std_PP_2_AccHigh", "Std_PP_3_AccHigh",
"Std_PP_2_AccLow", "Std_PP_3_AccLow"
)
combinedDf$Subject <- paste0("#", str_pad(combinedDf$Subject, 2, pad="0"))
combinedDf$ActivityEncoded <- factor(ifelse(combinedDf$Activity == "NO", "1", ifelse(combinedDf$Activity == "C", "2", "3")))
# combinedDf$PP_Dev_2_Turning <- ifelse(combinedDf$PP_Dev_2_Turning > 0, combinedDf$PP_Dev_2_Turning, combinedDf$PP_Dev_2_Straight)
combinedDf_NoStressor <- combinedDf[combinedDf$Activity == "NO",]
combinedDf_Cognitive <- combinedDf[combinedDf$Activity == "C",]
combinedDf_Motoric <- combinedDf[combinedDf$Activity == "M",]
combinedDf_NoStressor$Subject <- as.factor(combinedDf_NoStressor$Subject)
combinedDf_Cognitive$Subject <- as.factor(combinedDf_Cognitive$Subject)
combinedDf_Motoric$Subject <- as.factor(combinedDf_Motoric$Subject)
COLOR_NORMAL <- list(color='rgb(120,120,120)')
COLOR_COGNITIVE <- list(color='rgb(158,202,225)')
COLOR_MOTORIC <- list(color='rgb(58,200,225)')
COLOR_FAILURE_PRIOR <- list(color='green')
COLOR_FAILURE <- list(color='red')
COLOR_ACC <- list(color="orange")
yAxis <- list(
title = 'Perinasal Perspiration (Log)',
range=c(-0.3, 0.5)
)
# Apply Otsu algorithm to select threshold
ppDev <- combinedDf$PP_After # PP_Dev
ppDevArray <- matrix(ppDev, nrow = 1,ncol = length(ppDev))
THRESHOLD_MILD = otsu(ppDevArray, range=c(min(ppDev), max(ppDev))) # Expected Threshold > 0.042
print(paste0('Threshold: ', THRESHOLD_MILD))
[1] "Threshold: 0.101235546875"
MARKER_LINE_MILD = list(color="blue")
MARKER_LINE_EXTREME = list(color="red")
fig_NoStressor <- plot_ly(combinedDf_NoStressor, x = ~Subject, y = ~PP_Dev_2_Straight, type = 'bar', name = 'Cognitive - Mean PP (Straight)', marker=COLOR_COGNITIVE) %>%
# add_trace(y = ~PP_Dev_1_Turning, name = 'Normal - Mean PP (Turning)', marker=COLOR_COGNITIVE) %>%
add_trace(y = ~PP_Dev_2_Turning, name = 'Cognitive - Mean PP (Turning)', marker=COLOR_COGNITIVE) %>%
add_trace(y = ~Mean_PP_2_AccHigh, name = 'Coginitive - Mean PP (High Accel.)', marker=COLOR_ACC) %>%
add_trace(y = ~Mean_PP_2_AccLow, name = 'Coginitive - Mean PP (Low Accel.)', marker=COLOR_ACC) %>%
add_trace(y = ~PP_Dev_3_Straight, name = 'Motoric - Mean PP (Straight)', marker=COLOR_MOTORIC) %>%
add_trace(y = ~PP_Dev_3_Turning, name = 'Motoric - Mean PP (Turning)', marker=COLOR_MOTORIC) %>%
add_trace(y = ~Mean_PP_3_AccHigh, name = 'Motoric - Mean PP (High Accel.)', marker=COLOR_ACC) %>%
add_trace(y = ~Mean_PP_3_AccLow, name = 'Motoric - Mean PP (Low Accel.)', marker=COLOR_ACC) %>%
add_trace(y = ~PP_Prior, name = 'Failure - Prior PP', marker=COLOR_FAILURE_PRIOR) %>%
add_trace(y = ~PP_After, name = 'Failure - PP Deviation', marker=COLOR_FAILURE) %>%
add_segments(x="#01", xend="#41", y = THRESHOLD_MILD, yend = THRESHOLD_MILD, name="Threshold: Mild Change of PP",
line=list(color="blue", dash = 'dot')) %>%
# add_segments(x="#01", xend="#41", y = THRESHOLD_EXTREME, yend = THRESHOLD_EXTREME, name="Threshold: Extreme Change of PP",
# line=list(color="darkred", dash = 'dot')) %>%
layout(yaxis = yAxis, barmode = 'group', title="No Stressor")
htmltools::tagList(fig_NoStressor)
A marker object has been specified, but markers is not in the mode
Adding markers to the mode...
A marker object has been specified, but markers is not in the mode
Adding markers to the mode...
fig_Cognitive <- plot_ly(combinedDf_Cognitive, x = ~Subject, y = ~PP_Dev_2_Straight, type = 'bar', name = 'Cognitive - Mean PP (Straight)', marker=COLOR_COGNITIVE) %>%
add_trace(y = ~PP_Dev_2_Turning, name = 'Cognitive - Mean PP (Turning)', marker=COLOR_COGNITIVE) %>%
add_trace(y = ~Mean_PP_2_AccHigh, name = 'Coginitive - Mean PP (High Accel.)', marker=COLOR_ACC) %>%
add_trace(y = ~Mean_PP_2_AccLow, name = 'Coginitive - Mean PP (Low Accel.)', marker=COLOR_ACC) %>%
add_trace(y = ~PP_Dev_3_Straight, name = 'Motoric - Mean PP (Straight)', marker=COLOR_MOTORIC) %>%
add_trace(y = ~PP_Dev_3_Turning, name = 'Motoric - Mean PP (Turning)', marker=COLOR_MOTORIC) %>%
add_trace(y = ~Mean_PP_3_AccHigh, name = 'Motoric - Mean PP (High Accel.)', marker=COLOR_ACC) %>%
add_trace(y = ~Mean_PP_3_AccLow, name = 'Motoric - Mean PP (Low Accel.)', marker=COLOR_ACC) %>%
add_trace(y = ~PP_Prior, name = 'Failure - Prior PP', marker=COLOR_FAILURE_PRIOR) %>%
add_trace(y = ~PP_After, name = 'Failure - PP Deviation', marker=COLOR_FAILURE) %>%
add_segments(x="#02", xend="#22", y = THRESHOLD_MILD, yend = THRESHOLD_MILD, name="Threshold: Mild Change of PP",
line=list(color="blue", dash = 'dot')) %>%
# add_segments(x="#02", xend="#22", y = THRESHOLD_EXTREME, yend = THRESHOLD_EXTREME, name="Threshold: Extreme Change of PP",
# line=list(color="darkred", dash = 'dot')) %>%
layout(yaxis = yAxis, barmode = 'group', title="Stressor = Cognitive")
htmltools::tagList(fig_Cognitive)
A marker object has been specified, but markers is not in the mode
Adding markers to the mode...
A marker object has been specified, but markers is not in the mode
Adding markers to the mode...
fig_Motoric <- plot_ly(combinedDf_Motoric, x = ~Subject, y = ~PP_Dev_2_Straight, type = 'bar', name = 'Arousal in Drive C - Straight segment', marker=COLOR_COGNITIVE, width=870) %>%
add_trace(y = ~PP_Dev_2_Turning, name = 'Cognitive - Mean PP (Turning)', marker=COLOR_COGNITIVE) %>%
add_trace(y = ~Mean_PP_2_AccHigh, name = 'Coginitive - Mean PP (High Accel.)', marker=COLOR_ACC) %>%
add_trace(y = ~Mean_PP_2_AccLow, name = 'Coginitive - Mean PP (Low Accel.)', marker=COLOR_ACC) %>%
add_trace(y = ~PP_Dev_3_Straight, name = 'Motoric - Mean PP (Straight)', marker=COLOR_MOTORIC) %>%
add_trace(y = ~PP_Dev_3_Turning, name = 'Motoric - Mean PP (Turning)', marker=COLOR_MOTORIC) %>%
add_trace(y = ~Mean_PP_3_AccHigh, name = 'Motoric - Mean PP (High Accel.)', marker=COLOR_ACC) %>%
add_trace(y = ~Mean_PP_3_AccLow, name = 'Motoric - Mean PP (Low Accel.)', marker=COLOR_ACC) %>%
add_trace(y = ~PP_Prior, name = 'Arousal in Drive F - Under prior stressor', marker=COLOR_FAILURE_PRIOR) %>%
add_trace(y = ~PP_After, name = 'Arousal in Drive F - Unintended acceleration', marker=COLOR_FAILURE) %>%
add_segments(x="#05", xend="#31", y = THRESHOLD_MILD, yend = THRESHOLD_MILD, name="Threshold",
line=list(color="blue", dash = 'dot')) %>%
# add_segments(x="#05", xend="#31", y = THRESHOLD_EXTREME, yend = THRESHOLD_EXTREME, name="Threshold: Extreme Change of PP",
# line=list(color="darkred", dash = 'dot')) %>%
layout(yaxis = yAxis, barmode = 'group', title="Stressor = Motoric")
htmltools::tagList(fig_Motoric)
A marker object has been specified, but markers is not in the mode
Adding markers to the mode...
A marker object has been specified, but markers is not in the mode
Adding markers to the mode...
library(nlme)
combinedDf$Subject = as.factor(combinedDf$Subject)
combinedDf$Activity = as.factor(combinedDf$Activity)
combinedDf$PP_Dev_Group = ifelse(combinedDf$PP_Dev > THRESHOLD_MILD, 1, 0)
Linear model with all variables
combinedDfNoOutlier <- combinedDf[combinedDf$Subject != "#05",]
linearModel1 <- lm(PP_After ~
+ PP_Dev_2_Straight
+ PP_Dev_3_Straight
+ PP_Dev_2_Turning
+ PP_Dev_3_Turning
+ Std_PP_2_Straight
+ Std_PP_3_Straight
+ Std_PP_2_Turning
+ Std_PP_3_Turning
+ PP_Prior
+ factor(ActivityEncoded),
data=combinedDf)
# anova(model)
summary(linearModel1)
Call:
lm(formula = PP_After ~ +PP_Dev_2_Straight + PP_Dev_3_Straight +
PP_Dev_2_Turning + PP_Dev_3_Turning + Std_PP_2_Straight +
Std_PP_3_Straight + Std_PP_2_Turning + Std_PP_3_Turning +
PP_Prior + factor(ActivityEncoded), data = combinedDf)
Residuals:
Min 1Q Median 3Q Max
-0.076664 -0.027277 -0.000867 0.021999 0.100001
Coefficients:
Estimate Std. Error t value Pr(>|t|)
(Intercept) -0.07132 0.08729 -0.817 0.4350
PP_Dev_2_Straight 0.71569 0.36574 1.957 0.0821 .
PP_Dev_3_Straight -0.81109 0.44442 -1.825 0.1013
PP_Dev_2_Turning -0.55309 0.44548 -1.242 0.2458
PP_Dev_3_Turning 0.66117 0.46801 1.413 0.1914
Std_PP_2_Straight 1.41894 1.36691 1.038 0.3263
Std_PP_3_Straight 1.27101 0.71461 1.779 0.1090
Std_PP_2_Turning -1.53955 1.68233 -0.915 0.3840
Std_PP_3_Turning 0.16636 1.14505 0.145 0.8877
PP_Prior 0.70952 0.25103 2.826 0.0198 *
factor(ActivityEncoded)2 0.04926 0.07400 0.666 0.5224
factor(ActivityEncoded)3 0.12239 0.05641 2.170 0.0582 .
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
Residual standard error: 0.0682 on 9 degrees of freedom
Multiple R-squared: 0.9037, Adjusted R-squared: 0.7859
F-statistic: 7.675 on 11 and 9 DF, p-value: 0.002452
plot(linearModel1)




linearModel1 <- lm(PP_After ~
Mean_PP_2_AccHigh
+ Mean_PP_2_AccLow
+ Mean_PP_3_AccHigh
+ Mean_PP_3_AccLow
+ Std_PP_2_AccHigh
+ Std_PP_2_AccLow
+ Std_PP_3_AccHigh
+ Std_PP_3_AccLow
# + PP_Prior
+ factor(ActivityEncoded),
data=combinedDf)
# anova(model)
summary(linearModel1)
Call:
lm(formula = PP_After ~ Mean_PP_2_AccHigh + Mean_PP_2_AccLow +
Mean_PP_3_AccHigh + Mean_PP_3_AccLow + Std_PP_2_AccHigh +
Std_PP_2_AccLow + Std_PP_3_AccHigh + Std_PP_3_AccLow + factor(ActivityEncoded),
data = combinedDf)
Residuals:
Min 1Q Median 3Q Max
-0.110015 -0.048043 0.009167 0.036551 0.082263
Coefficients:
Estimate Std. Error t value Pr(>|t|)
(Intercept) -0.36297 0.09880 -3.674 0.00429 **
Mean_PP_2_AccHigh 2.14482 0.63161 3.396 0.00682 **
Mean_PP_2_AccLow -1.74610 0.63007 -2.771 0.01974 *
Mean_PP_3_AccHigh 2.99003 0.78293 3.819 0.00338 **
Mean_PP_3_AccLow -2.40638 0.75184 -3.201 0.00948 **
Std_PP_2_AccHigh 5.27168 4.04124 1.304 0.22130
Std_PP_2_AccLow -4.34469 2.79301 -1.556 0.15087
Std_PP_3_AccHigh 0.67278 1.84262 0.365 0.72262
Std_PP_3_AccLow 3.14657 2.14416 1.468 0.17297
factor(ActivityEncoded)2 0.18930 0.05019 3.771 0.00365 **
factor(ActivityEncoded)3 0.13988 0.05308 2.635 0.02494 *
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
Residual standard error: 0.07638 on 10 degrees of freedom
Multiple R-squared: 0.8657, Adjusted R-squared: 0.7315
F-statistic: 6.449 on 10 and 10 DF, p-value: 0.003428
plot(linearModel1)




# Export the anova table
library(xtable)
lmCoeffs <- summary(linearModel1)$coefficients
lmAnova <- anova(linearModel1)
print(xtable(lmCoeffs, digits=c(0,5,5,5,5)))
% latex table generated in R 3.6.1 by xtable 1.8-4 package
% Sat Jul 11 03:50:32 2020
\begin{table}[ht]
\centering
\begin{tabular}{rrrrr}
\hline
& Estimate & Std. Error & t value & Pr($>$$|$t$|$) \\
\hline
(Intercept) & -0.36297 & 0.09880 & -3.67377 & 0.00429 \\
Mean\_PP\_2\_AccHigh & 2.14482 & 0.63161 & 3.39577 & 0.00682 \\
Mean\_PP\_2\_AccLow & -1.74610 & 0.63007 & -2.77127 & 0.01974 \\
Mean\_PP\_3\_AccHigh & 2.99003 & 0.78293 & 3.81901 & 0.00338 \\
Mean\_PP\_3\_AccLow & -2.40638 & 0.75184 & -3.20067 & 0.00948 \\
Std\_PP\_2\_AccHigh & 5.27168 & 4.04124 & 1.30447 & 0.22130 \\
Std\_PP\_2\_AccLow & -4.34469 & 2.79301 & -1.55556 & 0.15087 \\
Std\_PP\_3\_AccHigh & 0.67278 & 1.84262 & 0.36512 & 0.72262 \\
Std\_PP\_3\_AccLow & 3.14657 & 2.14416 & 1.46750 & 0.17297 \\
factor(ActivityEncoded)2 & 0.18930 & 0.05019 & 3.77139 & 0.00365 \\
factor(ActivityEncoded)3 & 0.13988 & 0.05308 & 2.63525 & 0.02494 \\
\hline
\end{tabular}
\end{table}
print(xtable(lmAnova), digits=c(0,5,5,5,5))
% latex table generated in R 3.6.1 by xtable 1.8-4 package
% Sat Jul 11 03:50:32 2020
\begin{table}[ht]
\centering
\begin{tabular}{lrrrrr}
\hline
& Df & Sum Sq & Mean Sq & F value & Pr($>$F) \\
\hline
Mean\_PP\_2\_AccHigh & 1 & 0.15 & 0.15 & 26.27 & 0.0004 \\
Mean\_PP\_2\_AccLow & 1 & 0.00 & 0.00 & 0.32 & 0.5845 \\
Mean\_PP\_3\_AccHigh & 1 & 0.01 & 0.01 & 1.09 & 0.3207 \\
Mean\_PP\_3\_AccLow & 1 & 0.06 & 0.06 & 11.06 & 0.0077 \\
Std\_PP\_2\_AccHigh & 1 & 0.00 & 0.00 & 0.06 & 0.8116 \\
Std\_PP\_2\_AccLow & 1 & 0.00 & 0.00 & 0.50 & 0.4940 \\
Std\_PP\_3\_AccHigh & 1 & 0.03 & 0.03 & 5.02 & 0.0490 \\
Std\_PP\_3\_AccLow & 1 & 0.02 & 0.02 & 3.14 & 0.1070 \\
factor(ActivityEncoded) & 2 & 0.10 & 0.05 & 8.51 & 0.0069 \\
Residuals & 10 & 0.06 & 0.01 & & \\
\hline
\end{tabular}
\end{table}
ppAfter <- combinedDf$PP_After
ppAfterArray <- matrix(ppAfter, nrow = 1,ncol = length(ppAfter))
thresholdPPAfter <- otsu(ppAfterArray, range=c(min(ppAfter), max(ppAfter))) # Expected Threshold > 0.042
print(paste0('Threshold: ', thresholdPPAfter))
[1] "Threshold: 0.101235546875"
selectedDf <- combinedDf %>% select(
"Subject", "Activity", "PP_After", "PP_Prior",
"Mean_PP_2_AccHigh", "Mean_PP_3_AccHigh",
"Mean_PP_2_AccLow", "Mean_PP_3_AccLow",
"Std_PP_2_AccHigh", "Std_PP_3_AccHigh",
"Std_PP_2_AccLow", "Std_PP_3_AccLow")
selectedDf$Subject <- NULL
selectedDf$Activity_NO <- ifelse(selectedDf$Activity == "NO", 1, 0)
selectedDf$Activity_C <- ifelse(selectedDf$Activity == "C", 1, 0)
selectedDf$Activity_M <- ifelse(selectedDf$Activity == "M", 1, 0)
selectedDf$Activity <- NULL
# selectedDf$PP_Dev_1_Turning <- NULL
# selectedDf$Std_PP_2_Straight <- NULL
# selectedDf$Std_PP_2_Turning <- NULL
# selectedDf$Std_PP_3_Straight <- NULL
# selectedDf$Std_PP_3_Turning <- NULL
#
# # According to Linear model
# selectedDf$PP_Dev_2_Straight <- abs(selectedDf$PP_Dev_2_Straight)
# selectedDf$PP_Dev_3_Straight <- abs(selectedDf$PP_Dev_3_Straight)
# selectedDf$PP_Dev_2_Turning <- abs(selectedDf$PP_Dev_2_Turning)
# selectedDf$PP_Dev_3_Turning <- abs(selectedDf$PP_Dev_3_Turning)
# selectedDf$PP_Prior <- abs(selectedDf$PP_Prior) # NULL
selectedDf$Class <- ifelse(selectedDf$PP_After >= thresholdPPAfter, T, F)
selectedDf$PP_After <- NULL
print(names(selectedDf))
[1] "PP_Prior" "Mean_PP_2_AccHigh" "Mean_PP_3_AccHigh" "Mean_PP_2_AccLow" "Mean_PP_3_AccLow" "Std_PP_2_AccHigh"
[7] "Std_PP_3_AccHigh" "Std_PP_2_AccLow" "Std_PP_3_AccLow" "Activity_NO" "Activity_C" "Activity_M"
[13] "Class"
# library(mefa)
# combinedDf <- rep(combinedDf, 10)
set.seed(43)
n_folds <- 3
params <- param <- list(objective = "binary:logistic",
booster = "gbtree",
eval_metric = "auc",
eta = 0.1,
max_depth = 10,
alpha = 1,
lambda = 0,
gamma = 0.45,
min_child_weight = 0.3,
subsample = 0.5,
colsample_bytree = 1)
# XGBoost Model
xgb_m <- xgb.cv( params = param,
data = as.matrix(selectedDf %>% select(-Class)) ,
label = selectedDf$Class,
nrounds = 100,
verbose = F,
prediction = T,
maximize = F, # Change this value to F will help to run with more itineration
nfold = n_folds,
metrics = c("auc", "error"),
early_stopping_rounds = 50,
stratified = T,
scale_pos_weight = 1)
# xgb_m$evaluation_log[xgb_m$best_iteration,"test_auc_mean"]
xgb_m$evaluation_log[xgb_m$best_iteration,]
NA
Performance Metrics
# Prediction
selectedDf$clsPred <- round(xgb_m$pred)
computePerformanceResults <- function(sdat){
sdat = sdat[complete.cases(sdat),]
acc = sum(sdat[,1] == sdat[,2])/nrow(sdat)
conf_mat = table(sdat)
specif = conf_mat[1,1]/sum(conf_mat[,1])
sensiv = conf_mat[2,2]/sum(conf_mat[,2])
preci = conf_mat[2,2]/sum(conf_mat[2,])
npv = conf_mat[1,1]/sum(conf_mat[1,])
return(c(acc,specif,sensiv,preci,npv))
}
# Get average performance
performance <- computePerformanceResults(selectedDf %>% select(Class, clsPred))
acc <- performance[1]
prec <- performance[4]
recall <- performance[3]
spec <- performance[2]
npv <- performance[5]
f1 <- (2 * recall * prec) / (recall + prec)
auc <- as.numeric(xgb_m$evaluation_log[xgb_m$best_iteration, "test_auc_mean"])
print(paste("Accuracy=", round(acc, 2)))
[1] "Accuracy= 0.71"
print(paste("Precision=", round(prec, 2)))
[1] "Precision= 0.33"
print(paste("Recall=", round(recall, 2)))
[1] "Recall= 1"
print(paste("Specificity=", round(spec, 2)))
[1] "Specificity= 0.67"
print(paste("NPV=", round(npv, 2)))
[1] "NPV= 1"
print(paste("F1=", round(f1, 2)))
[1] "F1= 0.5"
print(paste("AUC=", round(auc, 2)))
[1] "AUC= 0.65"
# Importance
bst <- xgboost( params = param,
data = as.matrix(selectedDf %>% select(-c(Class, clsPred))) ,
label = selectedDf$Class,
nrounds = 100,
verbose = F,
prediction = T,
maximize = F, # Change this value to F will help to run with more itineration
nfold = n_folds,
metrics = c("auc", "error"),
early_stopping_rounds = 50,
stratified = T,
scale_pos_weight = 1)
importanceDf <- xgb.importance(colnames(selectedDf %>% select(-c(Class, clsPred))), model = bst)
print(importanceDf)
library(pROC)
dfROC <- pROC::roc(response = ifelse(selectedDf$Class==T, 1, 0),
predictor = round(xgb_m$pred),
levels=c(0, 1), direction = "<")
# it = which.max(xgb_m$evaluation_log$test_auc_mean)
# best.iter = xgb_m$evaluation_log$iter[it]
# best.iter
plot(pROC::roc(response = ifelse(selectedDf$Class==T, 1, 0),
predictor = round(xgb_m$pred),
levels=c(0, 1), direction = "<"),
legacy.axes = TRUE,
main="ROC Curve",
lwd=1.5)

Plot feature importance
yAxis <- list(
title = 'Importance',
range=c(0.0, 1.0)
)
xAxis <- list(
title = ''
)
importanceDf$Feature <- factor(importanceDf$Feature, levels = importanceDf[order(-Gain),]$Feature)
fig_Importance <- plot_ly(importanceDf, x = ~Feature, y = ~Gain, type = 'bar', name = 'Gain', width=600) %>%
add_trace(y = ~Cover, name = 'Cover') %>%
add_trace(y = ~Frequency, name = 'Frequency') %>%
layout(yaxis = yAxis, xaxis=xAxis, barmode = 'group', title="Feature Importance") %>%
config(.Last.value, mathjax = 'cdn')
htmltools::tagList(fig_Importance)
LS0tCnRpdGxlOiAiUiBOb3RlYm9vayIKb3V0cHV0OiBodG1sX25vdGVib29rCi0tLQoKYGBge3J9CnNvdXJjZSgnLi4vc2V0dGluZ3Mvc2V0dGluZ3MuUicpCnNvdXJjZSgnY29tbW9uRnVuY3Rpb25zLlInKQpgYGAKCmBgYHtyfQpkcml2ZTEgPC0gcmVhZC5jc3YoJy4uL2RhdGEvcHJvY2Vzc2VkL2FuYWx5c2lzL1RUMV9Ecml2ZV8xX1BQLmNzdicpCmRyaXZlMiA8LSByZWFkLmNzdignLi4vZGF0YS9wcm9jZXNzZWQvQW5hbHlzaXMvVFQxX0RyaXZlXzJfUFAuY3N2JykKZHJpdmUzIDwtIHJlYWQuY3N2KCcuLi9kYXRhL3Byb2Nlc3NlZC9BbmFseXNpcy9UVDFfRHJpdmVfM19QUC5jc3YnKQpkcml2ZTQgPC0gcmVhZC5jc3YoJy4uL2RhdGEvcHJvY2Vzc2VkL0FuYWx5c2lzL1RUMV9Ecml2ZV80X1BQLmNzdicsIHN0cmluZ3NBc0ZhY3RvcnMgPSBUKQpgYGAKCmBgYHtyfQpzZXQuc2VlZCg0MykKY29tYmluZWREZiA8LSBjYmluZChkcml2ZTQsIAogICAgICAgICAgICAgICAgICAgIGRyaXZlMSRNZWFuUFBfU2VnMCwgCiAgICAgICAgICAgICAgICAgICAgZHJpdmUyJE1lYW5QUCwgZHJpdmUzJE1lYW5QUCwKICAgICAgICAgICAgICAgICAgICBkcml2ZTIkU3RkUFAsIGRyaXZlMyRTdGRQUCwKICAgICAgICAgICAgICAgICAgICBkcml2ZTIkTWVhblBQX1NlZ01heCwgZHJpdmUzJE1lYW5QUF9TZWdNYXgsIAogICAgICAgICAgICAgICAgICAgIGRyaXZlMiRNZWFuUFBfU2VnMCwgZHJpdmUzJE1lYW5QUF9TZWcwLAogICAgICAgICAgICAgICAgICAgIGRyaXZlMiRTdGRQUF9TZWdNYXgsIGRyaXZlMyRTdGRQUF9TZWdNYXgsIAogICAgICAgICAgICAgICAgICAgIGRyaXZlMiRTdGRQUF9TZWcwLCBkcml2ZTMkU3RkUFBfU2VnMCwKICAgICAgICAgICAgICAgICAgICBkcml2ZTIkTWVhblBQX0FjY0hpZ2gsIGRyaXZlMyRNZWFuUFBfQWNjSGlnaCwKICAgICAgICAgICAgICAgICAgICBkcml2ZTIkWC5NZWFuUFBfQWNjTG93LCBkcml2ZTMkWC5NZWFuUFBfQWNjTG93LAogICAgICAgICAgICAgICAgICAgIGRyaXZlMiRTdGRQUF9BY2NIaWdoLCBkcml2ZTMkU3RkUFBfQWNjSGlnaCwKICAgICAgICAgICAgICAgICAgICBkcml2ZTIkU3RkUFBfQWNjTG93LCBkcml2ZTMkU3RkUFBfQWNjTG93CiAgICAgICAgICAgICAgICAgICkKbmFtZXMoY29tYmluZWREZikgPC0gYyhuYW1lcyhkcml2ZTQpLCAKICAgICAgICAgICAgICAgICAgICAgICAiUFBfRGV2XzFfVHVybmluZyIsCiAgICAgICAgICAgICAgICAgICAgICAgIlBQX0Rldl8yIiwgIlBQX0Rldl8zIiwgCiAgICAgICAgICAgICAgICAgICAgICAgIlN0ZF9QUF8yIiwgIlN0ZF9QUF8zIiwKICAgICAgICAgICAgICAgICAgICAgICAiUFBfRGV2XzJfU3RyYWlnaHQiLCAiUFBfRGV2XzNfU3RyYWlnaHQiLCAKICAgICAgICAgICAgICAgICAgICAgICAiUFBfRGV2XzJfVHVybmluZyIsICJQUF9EZXZfM19UdXJuaW5nIiwgCiAgICAgICAgICAgICAgICAgICAgICAgIlN0ZF9QUF8yX1N0cmFpZ2h0IiwgIlN0ZF9QUF8zX1N0cmFpZ2h0IiwgCiAgICAgICAgICAgICAgICAgICAgICAgIlN0ZF9QUF8yX1R1cm5pbmciLCAiU3RkX1BQXzNfVHVybmluZyIsCiAgICAgICAgICAgICAgICAgICAgICAgIk1lYW5fUFBfMl9BY2NIaWdoIiwgIk1lYW5fUFBfM19BY2NIaWdoIiwKICAgICAgICAgICAgICAgICAgICAgICAiTWVhbl9QUF8yX0FjY0xvdyIsICJNZWFuX1BQXzNfQWNjTG93IiwKICAgICAgICAgICAgICAgICAgICAgICAiU3RkX1BQXzJfQWNjSGlnaCIsICJTdGRfUFBfM19BY2NIaWdoIiwKICAgICAgICAgICAgICAgICAgICAgICAiU3RkX1BQXzJfQWNjTG93IiwgIlN0ZF9QUF8zX0FjY0xvdyIKICAgICAgICAgICAgICAgICAgICAgICApCgpjb21iaW5lZERmJFN1YmplY3QgPC0gcGFzdGUwKCIjIiwgc3RyX3BhZChjb21iaW5lZERmJFN1YmplY3QsIDIsIHBhZD0iMCIpKQpjb21iaW5lZERmJEFjdGl2aXR5RW5jb2RlZCA8LSBmYWN0b3IoaWZlbHNlKGNvbWJpbmVkRGYkQWN0aXZpdHkgPT0gIk5PIiwgIjEiLCBpZmVsc2UoY29tYmluZWREZiRBY3Rpdml0eSA9PSAiQyIsICIyIiwgIjMiKSkpCgojIGNvbWJpbmVkRGYkUFBfRGV2XzJfVHVybmluZyA8LSBpZmVsc2UoY29tYmluZWREZiRQUF9EZXZfMl9UdXJuaW5nID4gMCwgY29tYmluZWREZiRQUF9EZXZfMl9UdXJuaW5nLCBjb21iaW5lZERmJFBQX0Rldl8yX1N0cmFpZ2h0KQpgYGAKCmBgYHtyfQpjb21iaW5lZERmX05vU3RyZXNzb3IgPC0gY29tYmluZWREZltjb21iaW5lZERmJEFjdGl2aXR5ID09ICJOTyIsXQpjb21iaW5lZERmX0NvZ25pdGl2ZSA8LSBjb21iaW5lZERmW2NvbWJpbmVkRGYkQWN0aXZpdHkgPT0gIkMiLF0KY29tYmluZWREZl9Nb3RvcmljIDwtIGNvbWJpbmVkRGZbY29tYmluZWREZiRBY3Rpdml0eSA9PSAiTSIsXQoKY29tYmluZWREZl9Ob1N0cmVzc29yJFN1YmplY3QgPC0gYXMuZmFjdG9yKGNvbWJpbmVkRGZfTm9TdHJlc3NvciRTdWJqZWN0KQpjb21iaW5lZERmX0NvZ25pdGl2ZSRTdWJqZWN0IDwtIGFzLmZhY3Rvcihjb21iaW5lZERmX0NvZ25pdGl2ZSRTdWJqZWN0KQpjb21iaW5lZERmX01vdG9yaWMkU3ViamVjdCA8LSBhcy5mYWN0b3IoY29tYmluZWREZl9Nb3RvcmljJFN1YmplY3QpCmBgYAoKYGBge3J9CkNPTE9SX05PUk1BTCA8LSBsaXN0KGNvbG9yPSdyZ2IoMTIwLDEyMCwxMjApJykKQ09MT1JfQ09HTklUSVZFIDwtIGxpc3QoY29sb3I9J3JnYigxNTgsMjAyLDIyNSknKQpDT0xPUl9NT1RPUklDIDwtIGxpc3QoY29sb3I9J3JnYig1OCwyMDAsMjI1KScpCkNPTE9SX0ZBSUxVUkVfUFJJT1IgPC0gbGlzdChjb2xvcj0nZ3JlZW4nKQpDT0xPUl9GQUlMVVJFIDwtIGxpc3QoY29sb3I9J3JlZCcpCkNPTE9SX0FDQyA8LSBsaXN0KGNvbG9yPSJvcmFuZ2UiKQoKeUF4aXMgPC0gbGlzdCgKICB0aXRsZSA9ICdQZXJpbmFzYWwgUGVyc3BpcmF0aW9uIChMb2cpJywKICByYW5nZT1jKC0wLjMsIDAuNSkKKQoKIyBBcHBseSBPdHN1IGFsZ29yaXRobSB0byBzZWxlY3QgdGhyZXNob2xkCnBwRGV2IDwtIGNvbWJpbmVkRGYkUFBfQWZ0ZXIgIyBQUF9EZXYKcHBEZXZBcnJheSA8LSBtYXRyaXgocHBEZXYsIG5yb3cgPSAxLG5jb2wgPSBsZW5ndGgocHBEZXYpKQogIApUSFJFU0hPTERfTUlMRCA9IG90c3UocHBEZXZBcnJheSwgcmFuZ2U9YyhtaW4ocHBEZXYpLCBtYXgocHBEZXYpKSkgIyBFeHBlY3RlZCBUaHJlc2hvbGQgPiAwLjA0MgpwcmludChwYXN0ZTAoJ1RocmVzaG9sZDogJywgVEhSRVNIT0xEX01JTEQpKQoKTUFSS0VSX0xJTkVfTUlMRCA9IGxpc3QoY29sb3I9ImJsdWUiKQpNQVJLRVJfTElORV9FWFRSRU1FID0gbGlzdChjb2xvcj0icmVkIikKYGBgCgpgYGB7ciwgd2FybmluZz1GfQpmaWdfTm9TdHJlc3NvciA8LSBwbG90X2x5KGNvbWJpbmVkRGZfTm9TdHJlc3NvciwgeCA9IH5TdWJqZWN0LCB5ID0gflBQX0Rldl8yX1N0cmFpZ2h0LCB0eXBlID0gJ2JhcicsIG5hbWUgPSAnQ29nbml0aXZlIC0gTWVhbiBQUCAoU3RyYWlnaHQpJywgbWFya2VyPUNPTE9SX0NPR05JVElWRSkgJT4lCiAgIyBhZGRfdHJhY2UoeSA9IH5QUF9EZXZfMV9UdXJuaW5nLCBuYW1lID0gJ05vcm1hbCAtIE1lYW4gUFAgKFR1cm5pbmcpJywgbWFya2VyPUNPTE9SX0NPR05JVElWRSkgJT4lIAogIGFkZF90cmFjZSh5ID0gflBQX0Rldl8yX1R1cm5pbmcsIG5hbWUgPSAnQ29nbml0aXZlIC0gTWVhbiBQUCAoVHVybmluZyknLCBtYXJrZXI9Q09MT1JfQ09HTklUSVZFKSAlPiUgCiAgYWRkX3RyYWNlKHkgPSB+TWVhbl9QUF8yX0FjY0hpZ2gsIG5hbWUgPSAnQ29naW5pdGl2ZSAtIE1lYW4gUFAgKEhpZ2ggQWNjZWwuKScsIG1hcmtlcj1DT0xPUl9BQ0MpICU+JSAKICBhZGRfdHJhY2UoeSA9IH5NZWFuX1BQXzJfQWNjTG93LCBuYW1lID0gJ0NvZ2luaXRpdmUgLSBNZWFuIFBQIChMb3cgQWNjZWwuKScsIG1hcmtlcj1DT0xPUl9BQ0MpICU+JSAKICAKICBhZGRfdHJhY2UoeSA9IH5QUF9EZXZfM19TdHJhaWdodCwgbmFtZSA9ICdNb3RvcmljIC0gTWVhbiBQUCAoU3RyYWlnaHQpJywgbWFya2VyPUNPTE9SX01PVE9SSUMpICU+JSAKICBhZGRfdHJhY2UoeSA9IH5QUF9EZXZfM19UdXJuaW5nLCBuYW1lID0gJ01vdG9yaWMgLSBNZWFuIFBQIChUdXJuaW5nKScsIG1hcmtlcj1DT0xPUl9NT1RPUklDKSAlPiUgCiAgYWRkX3RyYWNlKHkgPSB+TWVhbl9QUF8zX0FjY0hpZ2gsIG5hbWUgPSAnTW90b3JpYyAtIE1lYW4gUFAgKEhpZ2ggQWNjZWwuKScsIG1hcmtlcj1DT0xPUl9BQ0MpICU+JSAKICBhZGRfdHJhY2UoeSA9IH5NZWFuX1BQXzNfQWNjTG93LCBuYW1lID0gJ01vdG9yaWMgLSBNZWFuIFBQIChMb3cgQWNjZWwuKScsIG1hcmtlcj1DT0xPUl9BQ0MpICU+JSAKICAKICBhZGRfdHJhY2UoeSA9IH5QUF9QcmlvciwgbmFtZSA9ICdGYWlsdXJlIC0gUHJpb3IgUFAnLCBtYXJrZXI9Q09MT1JfRkFJTFVSRV9QUklPUikgJT4lCiAgYWRkX3RyYWNlKHkgPSB+UFBfQWZ0ZXIsIG5hbWUgPSAnRmFpbHVyZSAtIFBQIERldmlhdGlvbicsIG1hcmtlcj1DT0xPUl9GQUlMVVJFKSAlPiUgCiAgYWRkX3NlZ21lbnRzKHg9IiMwMSIsIHhlbmQ9IiM0MSIsIHkgPSBUSFJFU0hPTERfTUlMRCwgeWVuZCA9IFRIUkVTSE9MRF9NSUxELCBuYW1lPSJUaHJlc2hvbGQ6IE1pbGQgQ2hhbmdlIG9mIFBQIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgbGluZT1saXN0KGNvbG9yPSJibHVlIiwgZGFzaCA9ICdkb3QnKSkgJT4lCiAgIyBhZGRfc2VnbWVudHMoeD0iIzAxIiwgeGVuZD0iIzQxIiwgeSA9IFRIUkVTSE9MRF9FWFRSRU1FLCB5ZW5kID0gVEhSRVNIT0xEX0VYVFJFTUUsIG5hbWU9IlRocmVzaG9sZDogRXh0cmVtZSBDaGFuZ2Ugb2YgUFAiLAogICMgICAgICAgICAgICAgICAgICAgICAgICAgIGxpbmU9bGlzdChjb2xvcj0iZGFya3JlZCIsIGRhc2ggPSAnZG90JykpICU+JQogIGxheW91dCh5YXhpcyA9IHlBeGlzLCBiYXJtb2RlID0gJ2dyb3VwJywgdGl0bGU9Ik5vIFN0cmVzc29yIikKCmh0bWx0b29sczo6dGFnTGlzdChmaWdfTm9TdHJlc3NvcikKYGBgCgpgYGB7ciwgd2FybmluZz1GfQpmaWdfQ29nbml0aXZlIDwtIHBsb3RfbHkoY29tYmluZWREZl9Db2duaXRpdmUsIHggPSB+U3ViamVjdCwgeSA9IH5QUF9EZXZfMl9TdHJhaWdodCwgdHlwZSA9ICdiYXInLCBuYW1lID0gJ0NvZ25pdGl2ZSAtIE1lYW4gUFAgKFN0cmFpZ2h0KScsIG1hcmtlcj1DT0xPUl9DT0dOSVRJVkUpICU+JQogIGFkZF90cmFjZSh5ID0gflBQX0Rldl8yX1R1cm5pbmcsIG5hbWUgPSAnQ29nbml0aXZlIC0gTWVhbiBQUCAoVHVybmluZyknLCBtYXJrZXI9Q09MT1JfQ09HTklUSVZFKSAlPiUgCiAgYWRkX3RyYWNlKHkgPSB+TWVhbl9QUF8yX0FjY0hpZ2gsIG5hbWUgPSAnQ29naW5pdGl2ZSAtIE1lYW4gUFAgKEhpZ2ggQWNjZWwuKScsIG1hcmtlcj1DT0xPUl9BQ0MpICU+JSAKICBhZGRfdHJhY2UoeSA9IH5NZWFuX1BQXzJfQWNjTG93LCBuYW1lID0gJ0NvZ2luaXRpdmUgLSBNZWFuIFBQIChMb3cgQWNjZWwuKScsIG1hcmtlcj1DT0xPUl9BQ0MpICU+JSAKICAKICBhZGRfdHJhY2UoeSA9IH5QUF9EZXZfM19TdHJhaWdodCwgbmFtZSA9ICdNb3RvcmljIC0gTWVhbiBQUCAoU3RyYWlnaHQpJywgbWFya2VyPUNPTE9SX01PVE9SSUMpICU+JSAKICBhZGRfdHJhY2UoeSA9IH5QUF9EZXZfM19UdXJuaW5nLCBuYW1lID0gJ01vdG9yaWMgLSBNZWFuIFBQIChUdXJuaW5nKScsIG1hcmtlcj1DT0xPUl9NT1RPUklDKSAlPiUgCiAgYWRkX3RyYWNlKHkgPSB+TWVhbl9QUF8zX0FjY0hpZ2gsIG5hbWUgPSAnTW90b3JpYyAtIE1lYW4gUFAgKEhpZ2ggQWNjZWwuKScsIG1hcmtlcj1DT0xPUl9BQ0MpICU+JSAKICBhZGRfdHJhY2UoeSA9IH5NZWFuX1BQXzNfQWNjTG93LCBuYW1lID0gJ01vdG9yaWMgLSBNZWFuIFBQIChMb3cgQWNjZWwuKScsIG1hcmtlcj1DT0xPUl9BQ0MpICU+JSAKICAKICBhZGRfdHJhY2UoeSA9IH5QUF9QcmlvciwgbmFtZSA9ICdGYWlsdXJlIC0gUHJpb3IgUFAnLCBtYXJrZXI9Q09MT1JfRkFJTFVSRV9QUklPUikgJT4lCiAgYWRkX3RyYWNlKHkgPSB+UFBfQWZ0ZXIsIG5hbWUgPSAnRmFpbHVyZSAtIFBQIERldmlhdGlvbicsIG1hcmtlcj1DT0xPUl9GQUlMVVJFKSAlPiUgCiAgYWRkX3NlZ21lbnRzKHg9IiMwMiIsIHhlbmQ9IiMyMiIsIHkgPSBUSFJFU0hPTERfTUlMRCwgeWVuZCA9IFRIUkVTSE9MRF9NSUxELCBuYW1lPSJUaHJlc2hvbGQ6IE1pbGQgQ2hhbmdlIG9mIFBQIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgbGluZT1saXN0KGNvbG9yPSJibHVlIiwgZGFzaCA9ICdkb3QnKSkgJT4lCiAgIyBhZGRfc2VnbWVudHMoeD0iIzAyIiwgeGVuZD0iIzIyIiwgeSA9IFRIUkVTSE9MRF9FWFRSRU1FLCB5ZW5kID0gVEhSRVNIT0xEX0VYVFJFTUUsIG5hbWU9IlRocmVzaG9sZDogRXh0cmVtZSBDaGFuZ2Ugb2YgUFAiLAogICMgICAgICAgICAgICAgICAgICAgICAgICAgIGxpbmU9bGlzdChjb2xvcj0iZGFya3JlZCIsIGRhc2ggPSAnZG90JykpICU+JQogIGxheW91dCh5YXhpcyA9IHlBeGlzLCBiYXJtb2RlID0gJ2dyb3VwJywgdGl0bGU9IlN0cmVzc29yID0gQ29nbml0aXZlIikKCmh0bWx0b29sczo6dGFnTGlzdChmaWdfQ29nbml0aXZlKQpgYGAKCgoKYGBge3IsIHdhcm5pbmc9Rn0KZmlnX01vdG9yaWMgPC0gcGxvdF9seShjb21iaW5lZERmX01vdG9yaWMsIHggPSB+U3ViamVjdCwgeSA9IH5QUF9EZXZfMl9TdHJhaWdodCwgdHlwZSA9ICdiYXInLCBuYW1lID0gJ0Fyb3VzYWwgaW4gRHJpdmUgQyAtIFN0cmFpZ2h0IHNlZ21lbnQnLCBtYXJrZXI9Q09MT1JfQ09HTklUSVZFLCB3aWR0aD04NzApICU+JQogIGFkZF90cmFjZSh5ID0gflBQX0Rldl8yX1R1cm5pbmcsIG5hbWUgPSAnQ29nbml0aXZlIC0gTWVhbiBQUCAoVHVybmluZyknLCBtYXJrZXI9Q09MT1JfQ09HTklUSVZFKSAlPiUgCiAgYWRkX3RyYWNlKHkgPSB+TWVhbl9QUF8yX0FjY0hpZ2gsIG5hbWUgPSAnQ29naW5pdGl2ZSAtIE1lYW4gUFAgKEhpZ2ggQWNjZWwuKScsIG1hcmtlcj1DT0xPUl9BQ0MpICU+JSAKICBhZGRfdHJhY2UoeSA9IH5NZWFuX1BQXzJfQWNjTG93LCBuYW1lID0gJ0NvZ2luaXRpdmUgLSBNZWFuIFBQIChMb3cgQWNjZWwuKScsIG1hcmtlcj1DT0xPUl9BQ0MpICU+JSAKICAKICBhZGRfdHJhY2UoeSA9IH5QUF9EZXZfM19TdHJhaWdodCwgbmFtZSA9ICdNb3RvcmljIC0gTWVhbiBQUCAoU3RyYWlnaHQpJywgbWFya2VyPUNPTE9SX01PVE9SSUMpICU+JSAKICBhZGRfdHJhY2UoeSA9IH5QUF9EZXZfM19UdXJuaW5nLCBuYW1lID0gJ01vdG9yaWMgLSBNZWFuIFBQIChUdXJuaW5nKScsIG1hcmtlcj1DT0xPUl9NT1RPUklDKSAlPiUgCiAgYWRkX3RyYWNlKHkgPSB+TWVhbl9QUF8zX0FjY0hpZ2gsIG5hbWUgPSAnTW90b3JpYyAtIE1lYW4gUFAgKEhpZ2ggQWNjZWwuKScsIG1hcmtlcj1DT0xPUl9BQ0MpICU+JSAKICBhZGRfdHJhY2UoeSA9IH5NZWFuX1BQXzNfQWNjTG93LCBuYW1lID0gJ01vdG9yaWMgLSBNZWFuIFBQIChMb3cgQWNjZWwuKScsIG1hcmtlcj1DT0xPUl9BQ0MpICU+JSAKICAKICBhZGRfdHJhY2UoeSA9IH5QUF9QcmlvciwgbmFtZSA9ICdBcm91c2FsIGluIERyaXZlIEYgLSBVbmRlciBwcmlvciBzdHJlc3NvcicsIG1hcmtlcj1DT0xPUl9GQUlMVVJFX1BSSU9SKSAlPiUKICBhZGRfdHJhY2UoeSA9IH5QUF9BZnRlciwgbmFtZSA9ICdBcm91c2FsIGluIERyaXZlIEYgLSBVbmludGVuZGVkIGFjY2VsZXJhdGlvbicsIG1hcmtlcj1DT0xPUl9GQUlMVVJFKSAlPiUgCiAgYWRkX3NlZ21lbnRzKHg9IiMwNSIsIHhlbmQ9IiMzMSIsIHkgPSBUSFJFU0hPTERfTUlMRCwgeWVuZCA9IFRIUkVTSE9MRF9NSUxELCBuYW1lPSJUaHJlc2hvbGQiLAogICAgICAgICAgICAgICAgICAgICAgICAgICBsaW5lPWxpc3QoY29sb3I9ImJsdWUiLCBkYXNoID0gJ2RvdCcpKSAlPiUKICAjIGFkZF9zZWdtZW50cyh4PSIjMDUiLCB4ZW5kPSIjMzEiLCB5ID0gVEhSRVNIT0xEX0VYVFJFTUUsIHllbmQgPSBUSFJFU0hPTERfRVhUUkVNRSwgbmFtZT0iVGhyZXNob2xkOiBFeHRyZW1lIENoYW5nZSBvZiBQUCIsCiAgIyAgICAgICAgICAgICAgICAgICAgICAgICAgbGluZT1saXN0KGNvbG9yPSJkYXJrcmVkIiwgZGFzaCA9ICdkb3QnKSkgJT4lCiAgbGF5b3V0KHlheGlzID0geUF4aXMsIGJhcm1vZGUgPSAnZ3JvdXAnLCB0aXRsZT0iU3RyZXNzb3IgPSBNb3RvcmljIikKCmh0bWx0b29sczo6dGFnTGlzdChmaWdfTW90b3JpYykKYGBgCgoKYGBge3J9CmxpYnJhcnkobmxtZSkKY29tYmluZWREZiRTdWJqZWN0ID0gYXMuZmFjdG9yKGNvbWJpbmVkRGYkU3ViamVjdCkKY29tYmluZWREZiRBY3Rpdml0eSA9IGFzLmZhY3Rvcihjb21iaW5lZERmJEFjdGl2aXR5KQpjb21iaW5lZERmJFBQX0Rldl9Hcm91cCA9IGlmZWxzZShjb21iaW5lZERmJFBQX0RldiA+IFRIUkVTSE9MRF9NSUxELCAxLCAwKQpgYGAKCiMjIyBFeHRyYWN0IGRhdGEgZm9yIGltcG9ydGFudCBmZWF0dXJlcwpgYGB7cn0KaW1wb3J0YW50RmVhdHVyZXNEZiA8LSBjb21iaW5lZERmICU+JSBzZWxlY3QoU3ViamVjdCwgU3RkX1BQXzMsIFBQX0Rldl8yX1R1cm5pbmcsIEFjdGl2aXR5LCBQUF9EZXYsIFBQX0Rldl9Hcm91cCkKYGBgCgojIExpbmVhciBtb2RlbCB3aXRoIGFsbCB2YXJpYWJsZXMKYGBge3J9CmNvbWJpbmVkRGZOb091dGxpZXIgPC0gY29tYmluZWREZltjb21iaW5lZERmJFN1YmplY3QgIT0gIiMwNSIsXQpsaW5lYXJNb2RlbDEgPC0gbG0oUFBfQWZ0ZXIgfiAKICAgICAgICAgICAgICArIFBQX0Rldl8yX1N0cmFpZ2h0CiAgICAgICAgICAgICAgKyBQUF9EZXZfM19TdHJhaWdodAogICAgICAgICAgICAgICsgUFBfRGV2XzJfVHVybmluZwogICAgICAgICAgICAgICsgUFBfRGV2XzNfVHVybmluZwogICAgICAgICAgICAgICsgU3RkX1BQXzJfU3RyYWlnaHQKICAgICAgICAgICAgICArIFN0ZF9QUF8zX1N0cmFpZ2h0CiAgICAgICAgICAgICAgKyBTdGRfUFBfMl9UdXJuaW5nCiAgICAgICAgICAgICAgKyBTdGRfUFBfM19UdXJuaW5nCiAgICAgICAgICAgICAgKyBQUF9QcmlvcgogICAgICAgICAgICAgICsgZmFjdG9yKEFjdGl2aXR5RW5jb2RlZCksIAogICAgICAgICAgICBkYXRhPWNvbWJpbmVkRGYpCgojIGFub3ZhKG1vZGVsKQpzdW1tYXJ5KGxpbmVhck1vZGVsMSkKcGxvdChsaW5lYXJNb2RlbDEpCmBgYAoKYGBge3J9CmxpbmVhck1vZGVsMSA8LSBsbShQUF9BZnRlciB+IAogICAgICAgICAgICAgICAgTWVhbl9QUF8yX0FjY0hpZ2gKICAgICAgICAgICAgICArIE1lYW5fUFBfMl9BY2NMb3cKICAgICAgICAgICAgICArIE1lYW5fUFBfM19BY2NIaWdoCiAgICAgICAgICAgICAgKyBNZWFuX1BQXzNfQWNjTG93CiAgICAgICAgICAgICAgKyBTdGRfUFBfMl9BY2NIaWdoCiAgICAgICAgICAgICAgKyBTdGRfUFBfMl9BY2NMb3cKICAgICAgICAgICAgICArIFN0ZF9QUF8zX0FjY0hpZ2gKICAgICAgICAgICAgICArIFN0ZF9QUF8zX0FjY0xvdwogICAgICAgICAgICAgICMgKyBQUF9QcmlvcgogICAgICAgICAgICAgICsgZmFjdG9yKEFjdGl2aXR5RW5jb2RlZCksIAogICAgICAgICAgICBkYXRhPWNvbWJpbmVkRGYpCgojIGFub3ZhKG1vZGVsKQpzdW1tYXJ5KGxpbmVhck1vZGVsMSkKcGxvdChsaW5lYXJNb2RlbDEpCmBgYAoKCmBgYHtyfQojIEV4cG9ydCB0aGUgYW5vdmEgdGFibGUKbGlicmFyeSh4dGFibGUpCmxtQ29lZmZzIDwtIHN1bW1hcnkobGluZWFyTW9kZWwxKSRjb2VmZmljaWVudHMKbG1Bbm92YSA8LSBhbm92YShsaW5lYXJNb2RlbDEpCgpwcmludCh4dGFibGUobG1Db2VmZnMsIGRpZ2l0cz1jKDAsNSw1LDUsNSkpKQpwcmludCh4dGFibGUobG1Bbm92YSksIGRpZ2l0cz1jKDAsNSw1LDUsNSkpCgpgYGAKCgpgYGB7cn0KcHBBZnRlciA8LSBjb21iaW5lZERmJFBQX0FmdGVyCnBwQWZ0ZXJBcnJheSA8LSBtYXRyaXgocHBBZnRlciwgbnJvdyA9IDEsbmNvbCA9IGxlbmd0aChwcEFmdGVyKSkKICAKdGhyZXNob2xkUFBBZnRlciA8LSBvdHN1KHBwQWZ0ZXJBcnJheSwgcmFuZ2U9YyhtaW4ocHBBZnRlciksIG1heChwcEFmdGVyKSkpICMgRXhwZWN0ZWQgVGhyZXNob2xkID4gMC4wNDIKcHJpbnQocGFzdGUwKCdUaHJlc2hvbGQ6ICcsIHRocmVzaG9sZFBQQWZ0ZXIpKQoKc2VsZWN0ZWREZiA8LSBjb21iaW5lZERmICU+JSBzZWxlY3QoCiAgICAgICAgICAgICAgICAgICJTdWJqZWN0IiwgIkFjdGl2aXR5IiwgIlBQX0FmdGVyIiwgIlBQX1ByaW9yIiwKICAgICAgICAgICAgICAgICAgIk1lYW5fUFBfMl9BY2NIaWdoIiwgIk1lYW5fUFBfM19BY2NIaWdoIiwKICAgICAgICAgICAgICAgICAgIk1lYW5fUFBfMl9BY2NMb3ciLCAiTWVhbl9QUF8zX0FjY0xvdyIsCiAgICAgICAgICAgICAgICAgICJTdGRfUFBfMl9BY2NIaWdoIiwgIlN0ZF9QUF8zX0FjY0hpZ2giLAogICAgICAgICAgICAgICAgICAiU3RkX1BQXzJfQWNjTG93IiwgIlN0ZF9QUF8zX0FjY0xvdyIpCgpzZWxlY3RlZERmJFN1YmplY3QgPC0gTlVMTApzZWxlY3RlZERmJEFjdGl2aXR5X05PIDwtIGlmZWxzZShzZWxlY3RlZERmJEFjdGl2aXR5ID09ICJOTyIsIDEsIDApCnNlbGVjdGVkRGYkQWN0aXZpdHlfQyA8LSBpZmVsc2Uoc2VsZWN0ZWREZiRBY3Rpdml0eSA9PSAiQyIsIDEsIDApCnNlbGVjdGVkRGYkQWN0aXZpdHlfTSA8LSBpZmVsc2Uoc2VsZWN0ZWREZiRBY3Rpdml0eSA9PSAiTSIsIDEsIDApCnNlbGVjdGVkRGYkQWN0aXZpdHkgPC0gTlVMTAoKIyBzZWxlY3RlZERmJFBQX0Rldl8xX1R1cm5pbmcgPC0gTlVMTAojIHNlbGVjdGVkRGYkU3RkX1BQXzJfU3RyYWlnaHQgPC0gTlVMTAojIHNlbGVjdGVkRGYkU3RkX1BQXzJfVHVybmluZyA8LSBOVUxMCiMgc2VsZWN0ZWREZiRTdGRfUFBfM19TdHJhaWdodCA8LSBOVUxMCiMgc2VsZWN0ZWREZiRTdGRfUFBfM19UdXJuaW5nIDwtIE5VTEwKIyAKIyAjIEFjY29yZGluZyB0byBMaW5lYXIgbW9kZWwKIyBzZWxlY3RlZERmJFBQX0Rldl8yX1N0cmFpZ2h0IDwtIGFicyhzZWxlY3RlZERmJFBQX0Rldl8yX1N0cmFpZ2h0KQojIHNlbGVjdGVkRGYkUFBfRGV2XzNfU3RyYWlnaHQgPC0gYWJzKHNlbGVjdGVkRGYkUFBfRGV2XzNfU3RyYWlnaHQpCiMgc2VsZWN0ZWREZiRQUF9EZXZfMl9UdXJuaW5nIDwtIGFicyhzZWxlY3RlZERmJFBQX0Rldl8yX1R1cm5pbmcpCiMgc2VsZWN0ZWREZiRQUF9EZXZfM19UdXJuaW5nIDwtIGFicyhzZWxlY3RlZERmJFBQX0Rldl8zX1R1cm5pbmcpCiMgc2VsZWN0ZWREZiRQUF9QcmlvciA8LSBhYnMoc2VsZWN0ZWREZiRQUF9QcmlvcikgIyBOVUxMCgpzZWxlY3RlZERmJENsYXNzIDwtIGlmZWxzZShzZWxlY3RlZERmJFBQX0FmdGVyID49IHRocmVzaG9sZFBQQWZ0ZXIsIFQsIEYpCnNlbGVjdGVkRGYkUFBfQWZ0ZXIgPC0gTlVMTAoKcHJpbnQobmFtZXMoc2VsZWN0ZWREZikpCmBgYAoKYGBge3J9CiMgbGlicmFyeShtZWZhKQojIGNvbWJpbmVkRGYgPC0gcmVwKGNvbWJpbmVkRGYsIDEwKSAKYGBgCgpgYGB7cn0Kc2V0LnNlZWQoNDMpCm5fZm9sZHMgPC0gMwpwYXJhbXMgPC0gcGFyYW0gPC0gbGlzdChvYmplY3RpdmUgICAgICAgPSAiYmluYXJ5OmxvZ2lzdGljIiwgCiAgICAgICAgICAgICAgIGJvb3N0ZXIgICAgICAgICAgPSAiZ2J0cmVlIiwKICAgICAgICAgICAgICAgZXZhbF9tZXRyaWMgICAgICA9ICJhdWMiLAogICAgICAgICAgICAgICBldGEgICAgICAgICAgICAgID0gMC4xLAogICAgICAgICAgICAgICBtYXhfZGVwdGggICAgICAgID0gMTAsCiAgICAgICAgICAgICAgIGFscGhhICAgICAgICAgICAgPSAxLAogICAgICAgICAgICAgICBsYW1iZGEgICAgICAgICAgID0gMCwKICAgICAgICAgICAgICAgZ2FtbWEgICAgICAgICAgICA9IDAuNDUsCiAgICAgICAgICAgICAgIG1pbl9jaGlsZF93ZWlnaHQgPSAwLjMsCiAgICAgICAgICAgICAgIHN1YnNhbXBsZSAgICAgICAgPSAwLjUsCiAgICAgICAgICAgICAgIGNvbHNhbXBsZV9ieXRyZWUgPSAxKQogICAgICAgICAgIAojIFhHQm9vc3QgTW9kZWwgICAgICAgICAKeGdiX20gPC0geGdiLmN2KCAgIHBhcmFtcyAgICAgICAgICAgICAgID0gcGFyYW0sCiAgICAgICAgICAgICAgICAgIGRhdGEgPSBhcy5tYXRyaXgoc2VsZWN0ZWREZiAlPiUgc2VsZWN0KC1DbGFzcykpICwKICAgICAgICAgICAgICAgICAgbGFiZWwgPSAgc2VsZWN0ZWREZiRDbGFzcywKICAgICAgICAgICAgICAgICAgbnJvdW5kcyAgICAgICAgICAgICA9IDEwMCwKICAgICAgICAgICAgICAgICAgdmVyYm9zZSAgICAgICAgICAgICA9IEYsCiAgICAgICAgICAgICAgICAgIHByZWRpY3Rpb24gICAgICAgICAgPSBULAogICAgICAgICAgICAgICAgICBtYXhpbWl6ZSAgICAgICAgICAgID0gRiwgIyBDaGFuZ2UgdGhpcyB2YWx1ZSB0byBGIHdpbGwgaGVscCB0byBydW4gd2l0aCBtb3JlIGl0aW5lcmF0aW9uCiAgICAgICAgICAgICAgICAgIG5mb2xkICAgICAgICAgICAgICAgPSBuX2ZvbGRzLAogICAgICAgICAgICAgICAgICBtZXRyaWNzICAgICAgICAgICAgID0gYygiYXVjIiwgImVycm9yIiksCiAgICAgICAgICAgICAgICAgIGVhcmx5X3N0b3BwaW5nX3JvdW5kcyA9IDUwLAogICAgICAgICAgICAgICAgICBzdHJhdGlmaWVkICAgICAgICAgICAgPSBULAogICAgICAgICAgICAgICAgICBzY2FsZV9wb3Nfd2VpZ2h0ICAgICAgPSAxKQoKIyB4Z2JfbSRldmFsdWF0aW9uX2xvZ1t4Z2JfbSRiZXN0X2l0ZXJhdGlvbiwidGVzdF9hdWNfbWVhbiJdCnhnYl9tJGV2YWx1YXRpb25fbG9nW3hnYl9tJGJlc3RfaXRlcmF0aW9uLF0KCmBgYAoKIyMgUGVyZm9ybWFuY2UgTWV0cmljcwpgYGB7cn0KIyBQcmVkaWN0aW9uCnNlbGVjdGVkRGYkY2xzUHJlZCA8LSByb3VuZCh4Z2JfbSRwcmVkKQoKY29tcHV0ZVBlcmZvcm1hbmNlUmVzdWx0cyA8LSBmdW5jdGlvbihzZGF0KXsKICBzZGF0ID0gc2RhdFtjb21wbGV0ZS5jYXNlcyhzZGF0KSxdCiAgYWNjID0gc3VtKHNkYXRbLDFdID09IHNkYXRbLDJdKS9ucm93KHNkYXQpCiAgY29uZl9tYXQgPSB0YWJsZShzZGF0KQogIHNwZWNpZiA9IGNvbmZfbWF0WzEsMV0vc3VtKGNvbmZfbWF0WywxXSkKICBzZW5zaXYgPSBjb25mX21hdFsyLDJdL3N1bShjb25mX21hdFssMl0pCiAgcHJlY2kgPSAgY29uZl9tYXRbMiwyXS9zdW0oY29uZl9tYXRbMixdKQogIG5wdiA9ICAgIGNvbmZfbWF0WzEsMV0vc3VtKGNvbmZfbWF0WzEsXSkKICByZXR1cm4oYyhhY2Msc3BlY2lmLHNlbnNpdixwcmVjaSxucHYpKQp9CgojIEdldCBhdmVyYWdlIHBlcmZvcm1hbmNlCnBlcmZvcm1hbmNlIDwtIGNvbXB1dGVQZXJmb3JtYW5jZVJlc3VsdHMoc2VsZWN0ZWREZiAlPiUgc2VsZWN0KENsYXNzLCBjbHNQcmVkKSkKYWNjIDwtIHBlcmZvcm1hbmNlWzFdCnByZWMgPC0gcGVyZm9ybWFuY2VbNF0KcmVjYWxsIDwtIHBlcmZvcm1hbmNlWzNdCnNwZWMgPC0gcGVyZm9ybWFuY2VbMl0KbnB2IDwtIHBlcmZvcm1hbmNlWzVdCmYxIDwtICgyICogcmVjYWxsICogcHJlYykgLyAocmVjYWxsICsgcHJlYykKYXVjIDwtIGFzLm51bWVyaWMoeGdiX20kZXZhbHVhdGlvbl9sb2dbeGdiX20kYmVzdF9pdGVyYXRpb24sICJ0ZXN0X2F1Y19tZWFuIl0pCgpwcmludChwYXN0ZSgiQWNjdXJhY3k9Iiwgcm91bmQoYWNjLCAyKSkpCnByaW50KHBhc3RlKCJQcmVjaXNpb249Iiwgcm91bmQocHJlYywgMikpKQpwcmludChwYXN0ZSgiUmVjYWxsPSIsIHJvdW5kKHJlY2FsbCwgMikpKQpwcmludChwYXN0ZSgiU3BlY2lmaWNpdHk9Iiwgcm91bmQoc3BlYywgMikpKQpwcmludChwYXN0ZSgiTlBWPSIsIHJvdW5kKG5wdiwgMikpKQpwcmludChwYXN0ZSgiRjE9Iiwgcm91bmQoZjEsIDIpKSkKcHJpbnQocGFzdGUoIkFVQz0iLCByb3VuZChhdWMsIDIpKSkKYGBgCgpgYGB7cn0KIyBJbXBvcnRhbmNlCmJzdCA8LSB4Z2Jvb3N0KCAgIHBhcmFtcyAgICAgICAgICAgICAgID0gcGFyYW0sCiAgICAgICAgICAgICAgICAgIGRhdGEgPSBhcy5tYXRyaXgoc2VsZWN0ZWREZiAlPiUgc2VsZWN0KC1jKENsYXNzLCBjbHNQcmVkKSkpICwKICAgICAgICAgICAgICAgICAgbGFiZWwgPSAgc2VsZWN0ZWREZiRDbGFzcywKICAgICAgICAgICAgICAgICAgbnJvdW5kcyAgICAgICAgICAgICA9IDEwMCwKICAgICAgICAgICAgICAgICAgdmVyYm9zZSAgICAgICAgICAgICA9IEYsCiAgICAgICAgICAgICAgICAgIHByZWRpY3Rpb24gICAgICAgICAgPSBULAogICAgICAgICAgICAgICAgICBtYXhpbWl6ZSAgICAgICAgICAgID0gRiwgIyBDaGFuZ2UgdGhpcyB2YWx1ZSB0byBGIHdpbGwgaGVscCB0byBydW4gd2l0aCBtb3JlIGl0aW5lcmF0aW9uCiAgICAgICAgICAgICAgICAgIG5mb2xkICAgICAgICAgICAgICAgPSBuX2ZvbGRzLAogICAgICAgICAgICAgICAgICBtZXRyaWNzICAgICAgICAgICAgID0gYygiYXVjIiwgImVycm9yIiksCiAgICAgICAgICAgICAgICAgIGVhcmx5X3N0b3BwaW5nX3JvdW5kcyA9IDUwLAogICAgICAgICAgICAgICAgICBzdHJhdGlmaWVkICAgICAgICAgICAgPSBULAogICAgICAgICAgICAgICAgICBzY2FsZV9wb3Nfd2VpZ2h0ICAgICAgPSAxKQppbXBvcnRhbmNlRGYgPC0geGdiLmltcG9ydGFuY2UoY29sbmFtZXMoc2VsZWN0ZWREZiAlPiUgc2VsZWN0KC1jKENsYXNzLCBjbHNQcmVkKSkpLCBtb2RlbCA9IGJzdCkKcHJpbnQoaW1wb3J0YW5jZURmKQpgYGAKCmBgYHtyfQpsaWJyYXJ5KHBST0MpCgpkZlJPQyA8LSBwUk9DOjpyb2MocmVzcG9uc2UgPSBpZmVsc2Uoc2VsZWN0ZWREZiRDbGFzcz09VCwgMSwgMCksCiAgICAgICAgICAgICAgIHByZWRpY3RvciA9IHJvdW5kKHhnYl9tJHByZWQpLAogICAgICAgICAgICAgICBsZXZlbHM9YygwLCAxKSwgZGlyZWN0aW9uID0gIjwiKQoKIyBpdCA9IHdoaWNoLm1heCh4Z2JfbSRldmFsdWF0aW9uX2xvZyR0ZXN0X2F1Y19tZWFuKQojIGJlc3QuaXRlciA9IHhnYl9tJGV2YWx1YXRpb25fbG9nJGl0ZXJbaXRdCiMgYmVzdC5pdGVyIAoKcGxvdChwUk9DOjpyb2MocmVzcG9uc2UgPSBpZmVsc2Uoc2VsZWN0ZWREZiRDbGFzcz09VCwgMSwgMCksCiAgICAgICAgICAgICAgIHByZWRpY3RvciA9IHJvdW5kKHhnYl9tJHByZWQpLAogICAgICAgICAgICAgICBsZXZlbHM9YygwLCAxKSwgZGlyZWN0aW9uID0gIjwiKSwgCiAgICAgbGVnYWN5LmF4ZXMgPSBUUlVFLAogICAgIG1haW49IlJPQyBDdXJ2ZSIsIAogICAgIGx3ZD0xLjUpIApgYGAKCgojIyMgUGxvdCBmZWF0dXJlIGltcG9ydGFuY2UKYGBge3J9CnlBeGlzIDwtIGxpc3QoCiAgdGl0bGUgPSAnSW1wb3J0YW5jZScsCiAgcmFuZ2U9YygwLjAsIDEuMCkKKQp4QXhpcyA8LSBsaXN0KAogIHRpdGxlID0gJycKKQoKaW1wb3J0YW5jZURmJEZlYXR1cmUgPC0gZmFjdG9yKGltcG9ydGFuY2VEZiRGZWF0dXJlLCBsZXZlbHMgPSBpbXBvcnRhbmNlRGZbb3JkZXIoLUdhaW4pLF0kRmVhdHVyZSkKZmlnX0ltcG9ydGFuY2UgPC0gcGxvdF9seShpbXBvcnRhbmNlRGYsIHggPSB+RmVhdHVyZSwgeSA9IH5HYWluLCB0eXBlID0gJ2JhcicsIG5hbWUgPSAnR2FpbicsIHdpZHRoPTYwMCkgJT4lCiAgYWRkX3RyYWNlKHkgPSB+Q292ZXIsIG5hbWUgPSAnQ292ZXInKSAlPiUgCiAgYWRkX3RyYWNlKHkgPSB+RnJlcXVlbmN5LCBuYW1lID0gJ0ZyZXF1ZW5jeScpICU+JSAKICBsYXlvdXQoeWF4aXMgPSB5QXhpcywgeGF4aXM9eEF4aXMsIGJhcm1vZGUgPSAnZ3JvdXAnLCB0aXRsZT0iRmVhdHVyZSBJbXBvcnRhbmNlIikgJT4lIAogIGNvbmZpZyguTGFzdC52YWx1ZSwgbWF0aGpheCA9ICdjZG4nKQoKaHRtbHRvb2xzOjp0YWdMaXN0KGZpZ19JbXBvcnRhbmNlKQpgYGAKCgo=